﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using Crawl;
using Crawl.Ext;
using HtmlAgilityPack;
using System.Text.RegularExpressions;
using System.IO;
using System.Threading;
using System.Net;
using System.Threading.Tasks;
using Util.Net.Http;
using Proxys.Checkers;

namespace Proxys
{
    public partial class ProxyLoaderForm : Form
    {
        private string SeedFile = @".\seeds.txt";

        public delegate void AddProxys(List<WebProxy> proxys);
        public ProxyLoaderForm()
        {
            InitializeComponent();
        }

        private Crawler NewCrawler()
        {
            Job job = new Job(new JobContext());
            Crawl.Crawler crawler = new Crawl.Crawler(job.Context);
            CrawlJobs.LoadJobFromFile(SeedFile, job.Context.Jobs.RootQueue);
            crawler.CrawlDelay = 100;
            crawler.MaxRequestCount = 1 << 30;
            crawler.MaxFailedCount = 20;

            ModuleHandler moduleHandler = new ModuleHandler(crawler);
            RegisterModule(moduleHandler, crawler);

            crawler.RegisterCrawlUrlHandler(moduleHandler);
            crawler.onStart += OnCrawlerStart;
            crawler.onStop += OnCrawlerStop;
           
            return crawler;
        }
        private void RegisterModule(ModuleHandler moduleHandler, Crawl.Crawler crawler)
        {
            moduleHandler.RegisterModule(new ModuleEx.M51Module(), new ModuleEx.M51Handler(crawler, HandleNewProxys));
            moduleHandler.RegisterModule(new ModuleEx.PachongModule(), new ModuleEx.PachongHandler(crawler, HandleNewProxys));
            moduleHandler.RegisterModule(new ModuleEx.YouDailiGuoneiListModule(), new ModuleEx.YouDailiGuoneiListHandler(crawler));
            moduleHandler.RegisterModule(new ModuleEx.YouDailiProxysPageModule(), new ModuleEx.YouDailiProxysPageHandler(crawler, HandleNewProxys));
            moduleHandler.RegisterModule(new ModuleEx.Proxy360Module(), new ModuleEx.Proxy360Handler(crawler, HandleNewProxys));
            moduleHandler.RegisterModule(new ModuleEx.ProxyRuModule(), new ModuleEx.ProxyRuHandler(crawler, HandleNewProxys));
            moduleHandler.RegisterModule(new ModuleEx.XiciModule(), new ModuleEx.XiciHandler(crawler, HandleNewProxys));
            moduleHandler.RegisterModule(new ModuleEx.QuickModule(), new ModuleEx.QuickHandler(crawler, HandleNewProxys));
            moduleHandler.RegisterModule(new ModuleEx.ProxyCnProxyModule(), new ModuleEx.ProxyCnProxyHandler(crawler, HandleNewProxys));
        }


        private void HandleNewProxys(List<WebProxy> proxys)
        {
            List<string> strings = new List<string>();
            foreach (WebProxy proxy in proxys)
            {
                strings.Add(string.Format("{0}:{1}", proxy.Address.Host, proxy.Address.Port));
            }
            lock (this)
            {
                File.AppendAllLines(this.loadProxyTxb.Text, strings);
            }
        }
        public void OnCrawlerStop(Crawler crawler)
        {
            Console.WriteLine("Damn Stopped");
            PrintLn("Damn Stopped");
        }
        public void OnCrawlerStart(Crawler crawler) 
        {
            Console.WriteLine("Begin .....");
            PrintLn("Begin .....");
        }
        class ProxyComparer : IEqualityComparer<WebProxy>
        {
            public bool Equals(WebProxy x, WebProxy y)
            {
                return x.Address.Host.Equals(y.Address.Host) && x.Address.Port == y.Address.Port;
            }
            public int GetHashCode(WebProxy obj)
            {
                return obj.Address.Host.GetHashCode() + obj.Address.Port;
            }
        }
        class ProxyHostComparer : IEqualityComparer<WebProxy>
        {
            public bool Equals(WebProxy x, WebProxy y)
            {
                return x.Address.Host.Equals(y.Address.Host);
            }
            public int GetHashCode(WebProxy obj)
            {
                return obj.Address.Host.GetHashCode();
            }
        }

        private List<WebProxy> LoadProxys(string file)
        {
            string[] lines = File.ReadAllLines(file);
           IEnumerable<string> distinctLines = lines.Distinct();
           return new List<WebProxy>(distinctLines.Select(Line2Proxy).Where(proxy => proxy != null));
        }
        private WebProxy Line2Proxy(string line)
        {
            if (string.IsNullOrWhiteSpace(line)) { return null; }
            string host = line.Split(':')[0];
            int port = Int32.Parse(line.Split(':')[1]);
            return new WebProxy(host, port);
        }
        private string Proxy2Line(WebProxy proxy)
        {
            return proxy.Address.Host + ":" + proxy.Address.Port;
        }

        #region 界面调整
        private void ProxyLoaderForm_Load(object sender, EventArgs e)
        {
            this.fileTxb_TextChanged(sender, e);
        }
        private void fileTxb_TextChanged(object sender, EventArgs e)
        {
            FileInfo fileInfo = new FileInfo(this.loadProxyTxb.Text);
            this.blackTxb.Text = fileInfo.Directory.FullName + @"\black-" + fileInfo.Name;
            this.bdTxb.Text = fileInfo.Directory.FullName + @"\bd-" + fileInfo.Name;
            this.anonymousTxb.Text = fileInfo.Directory.FullName + @"\anonymous-" + fileInfo.Name;
            this.jyTxb.Text = fileInfo.Directory.FullName + @"\jy-" + fileInfo.Name;
        }
        #endregion
        private void AsynRun(Action act)
        {
            new Thread(() =>
            {
                PrintLn("Start");
                act();
                PrintLn("End");
            }).Start();
        }
        private static string[] SITES = new string[] { 
            @"http://www.baidu.com", 
            @"http://www.qq.com/", 
            @"http://image.baidu.com/", 
            @"http://tieba.baidu.com/", 
            @"http://news.baidu.com/", 
            @"http://www.baidu.com/more/", 
            @"http://music.baidu.com/",
            @"http://music.baidu.com/mv",
            @"http://music.baidu.com/tag",
            @"http://lifenote.baidu.com/",
            @"http://jiaoyu.baidu.com/",
        };
        private void LoadProxyBtn_Click(object sender, EventArgs e)
        {
            delHisFiles();
            Crawler crawler = NewCrawler();
            crawler.onCrawl += new OnCrawl((c,str)=> { PrintLn(str); });
            crawler.onStop += new OnStop(c => { PrintLn("结束"); });
            crawler.AsynStart();
        }

        private void delHisFiles() 
        {
            string outputPath = @".\jobs\output";
            DirectoryInfo outputDir = new DirectoryInfo(outputPath);
            if (outputDir.Exists) {
                outputDir.Delete(true);
            }
            string savePath = @".\jobs\save";
            DirectoryInfo saveDir = new DirectoryInfo(savePath);
            if (saveDir.Exists)
            {
                saveDir.Delete(true);
            }
        }

        private void checkAnonymousLvBtn_Click(object sender, EventArgs e)
        {
            IEnumerable<WebProxy> proxys = LoadProxys(this.bdTxb.Text);
            int minLv = Int32.Parse(this.anonyLvCbx.Text);
            proxys = Helper.FilterProxysByAnonymousLv(proxys, minLv);
            File.Delete(this.anonymousTxb.Text);
            File.WriteAllLines(this.anonymousTxb.Text, proxys.Select(Proxy2Line), Encoding.UTF8);
        }
        
        static Random r = new Random();
        private void bdCheckBtn_Click(object sender, EventArgs e)
        {
            string[] testUrl = new string[] {this.txtTestUrl.Text };
           IEnumerable<string> urlsDistinct =  testUrl.Distinct();
            AsynRun(() =>
            {
                IEnumerable<WebProxy> proxys = LoadProxys(this.loadProxyTxb.Text);
                proxys = CheckProxys(proxys, urlsDistinct.ToArray());
                File.Delete(this.txtDesProxyPath.Text);
                File.WriteAllLines(this.txtDesProxyPath.Text, proxys.Select(Proxy2Line));
            });
        }
        private List<WebProxy> CheckProxys(IEnumerable<WebProxy> proxys, string[] tarUrls, Func<string, bool> checkLegal = null,string encode = null ,int delay = 10000)
        {
            proxys = proxys.Distinct(new ProxyComparer());
            int rNum = r.Next();
            proxys = proxys.OrderBy(proxy => proxy.GetHashCode() ^ rNum);
            this.Invoke(new Action<int>(num => { progressBar1.Maximum = num; progressBar1.Value = 0; }), proxys.Count());
            List<WebProxy> newProxys = new List<WebProxy>();
            ParallelOptions options = new ParallelOptions();
            options.MaxDegreeOfParallelism = tarUrls.Length * 8;
            Parallel.ForEach(proxys, options, proxy =>
            {
                try
                {
                    string testUrl = tarUrls[r.Next(tarUrls.Length)];
                    HttpClient client = Helper.BuildHttpClient();
                    client.AllowAutoRedirect = true;
                    if(encode !=null){
                        client.ResponseEncoding = Encoding.GetEncoding(encode);                        
                    }
                    client.Timeout = delay;
                    client.Proxy = proxy;
                    HttpWebResponse response = client.GetResponse(testUrl);
                    if (response.StatusCode == HttpStatusCode.OK)
                    {
                        string content = client.ResponseString(response);
                        if (checkLegal == null || checkLegal(content))
                        {
                            lock (newProxys)
                            {
                                PrintLn(Proxy2Line(proxy));
                                newProxys.Add(proxy);
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.ToString());
                }
                finally
                {
                    this.Invoke(new Action(() => { lock (this) { progressBar1.Value = progressBar1.Value + 1; } }));
                }
            });
            return newProxys;
        }
        private void PrintLn(string msg)
        {
            this.Invoke(new Action<string>(line => { this.outputTxb.AppendText(line + Environment.NewLine); }), msg);
        }
        private void jyBtn_Click(object sender, EventArgs e)
        {
            AsynRun(() =>
            {
                IEnumerable<WebProxy> proxys = LoadProxys(this.bdTxb.Text);
                proxys = CheckProxys(proxys, new string[] { @"http://www.jyeoo.com" },
                    content => content.Contains(@"菁优网")
                    );
                proxys = proxys.Distinct(new ProxyHostComparer());
                File.Delete(this.jyTxb.Text);
                File.WriteAllLines(this.jyTxb.Text, proxys.Select(Proxy2Line));
            });
        }

        private void button1_Click(object sender, EventArgs e)
        {
            this.bdBtn.PerformClick();
            this.jyBtn.PerformClick();
        }

        private void btnPcs_Click(object sender, EventArgs e)
        {
            PropertiesProxyChecker properties = PropertiesProxyChecker.ReadProperties(Configs.ProxyCheckerProperties);

            AsynRun(() =>
            {
                IEnumerable<WebProxy> proxys = LoadProxys(this.loadProxyTxb.Text);

                foreach (ProxyChecker pChecker in properties.ProxyCheckers)
                {

                    PrintLn(pChecker.FileName);
                    Console.WriteLine("代理数：" + proxys.Count());
                    IEnumerable<WebProxy> tmpProxys = null;
                    tmpProxys = CheckProxys(
                        proxys,
                        new string[] { pChecker.TestUrl },
                        new Func<string, bool>(content => { return !string.IsNullOrEmpty(content) && content.Contains(pChecker.Key); }),
                        pChecker.Encode,
                        pChecker.MaxDelayTime);
                    File.Delete(pChecker.FileName);
                    File.WriteAllLines("proxy\\"+pChecker.FileName, tmpProxys.Select(Proxy2Line));
                }
            });

        }
        private void btnCheck_Click(object sender, EventArgs e)
        {
            string[] testUrls = new string[]{txtTestUrl.Text};
            int maxDalayTime =Decimal.ToInt32( numDelayTime.Value);
            string strToBeContain = txtShouldContait.Text;
            AsynRun(() =>
            {
                IEnumerable<WebProxy> proxys = LoadProxys(this.loadProxyTxb.Text);
                
                Console.WriteLine("代理数："+proxys.Count());
                if (rdo365checker.Checked)
                {
                    proxys = CheckProxys(proxys, testUrls, new Func<string, bool>(checkProxy365), "GBK", maxDalayTime);
                }
                else 
                {
                    Func<string, bool> checkFunc = new Func<string, bool>(content =>
                    {
                        if (!string.IsNullOrEmpty(content))
                        {
                            return string.IsNullOrWhiteSpace(strToBeContain) ? true : content.Contains(strToBeContain);
                        }
                        return false;
                    });

                    proxys = CheckProxys(proxys, testUrls, checkFunc, null, maxDalayTime);
                }
                File.Delete(this.txtDesProxyPath.Text);
                File.WriteAllLines(this.txtDesProxyPath.Text, proxys.Select(Proxy2Line));
            });
        }

        private bool checkProxy365(string content)
        {
            if (!string.IsNullOrEmpty(content)) 
            {
                return content.Contains("IP用户");
            }
            return false;
        }

        private void btnOpenDir_Click(object sender, EventArgs e)
        {
            FileInfo fileInfo = new FileInfo (txtDesProxyPath.Text);
            DirectoryInfo dir = fileInfo.Directory;
            if (!dir.Exists)
            {
                dir.Create();
            }
            System.Diagnostics.Process.Start("explorer.exe", dir.FullName);
        }

      

      
    }


    #region 各种代理网页及其处理
    public class ModuleEx
    {
        #region kuaidaili
        public class QuickModule : HtmlModule
        {
            public override string ModuleName
            {
                get { return "kuaidaili"; }
            }

            public override bool Match(CrawlUrl url)
            {
                return url.Url.Contains(@"www.kuaidaili.com/free/intr") || url.Url.Contains(@"www.kuaidaili.com/free/inha");
            }
        }

        public class QuickHandler : ProxyLoaderHandlerBase
        {
            public QuickHandler(Crawler crawler, Proxys.ProxyLoaderForm.AddProxys addProxys)
                : base(crawler, addProxys)
            {
            }

            public override void DoParse(CrawlUrl url)
            {
                this.Crawler.FetchContent(url);
                HtmlNode root = this.LoadHtml(url.Content);
                HtmlNodeCollection nodes = root.SelectNodes(@".//table[@class='table table-bordered table-striped']/tbody/tr");
                if (nodes == null)
                { return; }
                List<WebProxy> proxys = new List<WebProxy>();
                foreach (HtmlNode node in nodes)
                {
                    string host = node.SelectSingleNode(@"td[1]").InnerText;
                    int port = Int32.Parse(node.SelectSingleNode(@"td[2]").InnerText);
                    if (!HostRegex.IsMatch(host)) { continue; }
                    proxys.Add(new WebProxy(host, port));
                }
                this.AddProxys(proxys);
                
                Regex rnum = new Regex(@"[0-9]+");
                Match m = rnum.Match(url.Url);
                HtmlNodeCollection pageNode = root.SelectNodes(@".//div[@id='listnav']/ul/li/a");
                string pagelLast = pageNode.Last().InnerText;
                if (m.Length > 0)
                {
                    try
                    {      
                        int page = Int32.Parse(m.Value) + 1;
                        if (Int32.Parse(pagelLast) < page)
                        {
                            return;
                        }
                        string newUrl = rnum.Replace(url.Url, page + "");
                        this.Crawler.MyJobQueue.AddLasts(new CrawlUrl(newUrl));
                    }
                    catch
                    {
                        Console.WriteLine("!");
                    }             
                } 
            }
        }
        #endregion

        #region xici
        public class XiciModule : HtmlModule
        {
            public override string ModuleName
            {
                get { return "cixi"; }
            }

            public override bool Match(CrawlUrl url)
            {
                return url.Url.Contains(@"xici.net.co/nn") || url.Url.Contains(@"xici.net.co/nt");
            }
        }

        public class XiciHandler : ProxyLoaderHandlerBase
        {
            public XiciHandler(Crawler crawler, Proxys.ProxyLoaderForm.AddProxys addProxys)
                : base(crawler, addProxys)
            {
            }

            public override void DoParse(CrawlUrl url)
            {
                this.Crawler.FetchContent(url);
                HtmlNode root = this.LoadHtml(url.Content);
                HtmlNodeCollection nodes = root.SelectNodes(@".//tr[@class='']");
                HtmlNodeCollection nodes2 = root.SelectNodes(@".//tr[@class='odd']");
                HtmlNode nextNode = root.SelectSingleNode(@".//a[@class='next_page']");
                if (nodes == null || nodes2 == null || nextNode == null) 
                {
                    return;
                }
                foreach (HtmlNode node in nodes2)
                {
                    nodes.Append(node);
                }
                if (nodes == null)
                { return; }
                List<WebProxy> proxys = new List<WebProxy>();
                foreach (HtmlNode node in nodes)
                {
                    string host = node.SelectSingleNode(@"td[2]").InnerText;
                    int port = Int32.Parse(node.SelectSingleNode(@"td[3]").InnerText);
                    string area = node.SelectSingleNode(@"td[4]").InnerText;
                    if (!HostRegex.IsMatch(host)) { continue; }
                    proxys.Add(new WebProxy(host, port));
                }
                this.AddProxys(proxys);

                if (nextNode != null)
                {
                    if (nextNode.Attributes.Contains("href"))
                    {
                        this.Crawler.MyJobQueue.AddLasts(new CrawlUrl("http://www.xici.net.co" + nextNode.Attributes["href"].Value));
                    }
                    else
                    {
                        Console.WriteLine("why there is no links;");
                    }
                }
            }
        }
        #endregion
        
        #region 51代理
        public class M51Module : HtmlModule
        {
            public override string ModuleName
            {
                get { return "51proxied"; }
            }

            public override bool Match(CrawlUrl url)
            {
                return url.Url.Contains(@"taobaofou.com/http");
            }
        }
        public class M51Handler : ProxyLoaderHandlerBase
        {
            public M51Handler(Crawler crawler, Proxys.ProxyLoaderForm.AddProxys addProxys)
                : base(crawler, addProxys)
            {
            }

            public override void DoParse(CrawlUrl url)
            {
                this.Crawler.FetchContent(url);
                HtmlNode root = this.LoadHtml(url.Content);
                HtmlNodeCollection nodes = root.SelectNodes(@"//div[@id='container']/div[@class='box']/div[@id='tb']/table/tr");
                if (nodes == null)
                { return; }
                List<WebProxy> proxys = new List<WebProxy>();
                foreach (HtmlNode node in nodes)
                {
                    HtmlNodeCollection tdNodes = node.SelectNodes("td");
                    if (tdNodes != null && tdNodes.Count >= 4)
                    {
                        string host = node.SelectSingleNode(@"td[2]").InnerText;
                        int port = Int32.Parse(node.SelectSingleNode(@"td[3]").InnerText);
                        string area = node.SelectSingleNode(@"td[4]").InnerText;
                        if (!HostRegex.IsMatch(host) || !area.Equals("CN")) { continue; }
                        proxys.Add(new WebProxy(host, port));
                    }
                }

                this.AddProxys(proxys);
            }
        }
        #endregion
        
        #region 爬虫网
        public class PachongModule : HtmlModule
        {
            public override string ModuleName
            {
                get { return "Pachong"; }
            }

            public override bool Match(CrawlUrl url)
            {
                return url.Url.ToLower().Contains(@"pachong.org") && url.Url.ToLower().Contains(@"cn");
            }
        }
        public class PachongHandler : ProxyLoaderHandlerBase
        {
            public PachongHandler(Crawler crawler, Proxys.ProxyLoaderForm.AddProxys addProxys)
                : base(crawler, addProxys) { }

            public override void DoParse(CrawlUrl url)
            {
                this.Crawler.FetchContent(url);
                HtmlNode root = this.LoadHtml(url.Content);
                HtmlNodeCollection nodes = root.SelectNodes(@"//div[@class='mainWap']/table[@class='tb']//tr");
                if (nodes == null)
                { return; }
                List<WebProxy> proxys = new List<WebProxy>();
                foreach (HtmlNode node in nodes)
                {
                    try
                    {
                        HtmlNodeCollection tdNodes = node.SelectNodes("td");
                        if (tdNodes != null && tdNodes.Count >= 4)
                        {
                            string host = node.SelectSingleNode(@"td[2]").InnerText;
                            int port = Int32.Parse(node.SelectSingleNode(@"td[3]").InnerText);
                            string area = node.SelectSingleNode(@"td[4]").InnerText;
                            if (!HostRegex.IsMatch(host) || !area.Contains("中国")) { continue; }
                            proxys.Add(new WebProxy(host, port));
                        }
                    }
                    catch (Exception e)
                    {
                        Context.Logger.Error("~~~~~~~~~~~~~~~~~~~" + Environment.NewLine + node.OuterHtml, e);
                    }
                }
                AddProxys(proxys);
            }
        }
        #endregion
         
        #region 有代理网
        public class YouDailiGuoneiListModule : RegexUrlModule
        {
            public override string ModuleName
            {
                get { return "有代理-国内-List页面"; }
            }

            public override string UrlRegexString()
            {
                return @"http://www.youdaili.net/Daili/guonei/list_[0-9]*.html";
            }
        }
        public class YouDailiGuoneiListHandler : LinkModuleHandle
        {
            public YouDailiGuoneiListHandler(Crawler crawler)
                : base(crawler) { }
            public override List<CrawlUrl> ParseLinks(CrawlUrl url, HtmlNode root)
            {
                List<CrawlUrl> links = new List<CrawlUrl>();
                HtmlNodeCollection htmlNodes = root.SelectNodes(@"//ul[@class='newslist_line']/li/a");
                if (htmlNodes != null && htmlNodes.Count > 0)
                {
                    foreach (HtmlNode htmlNode in htmlNodes)
                    {
                        if (htmlNode.Attributes.Contains("href"))
                        {
                            links.Add(new CrawlUrl(htmlNode.Attributes["href"].Value));
                        }
                        else
                        {
                            Console.WriteLine("why there is no links;");
                        }
                    }
                }
                else
                {
                    Console.WriteLine("cannot parse links:" + url.Url);
                }
                return links;
            }
        }
        public class YouDailiProxysPageModule : RegexUrlModule
        {
            public override string UrlRegexString()
            {
                return @"http://www.youdaili.net/Daili/[A-Za-z0-9_]*/[0-9]*[_0-9]*.html";
            }
            public override string ModuleName
            {
                get { return "有代理-代理详情页"; }
            }
        }
        public class YouDailiProxysPageHandler : ProxyLoaderHandlerBase
        {
            public YouDailiProxysPageHandler(Crawler crawler, Proxys.ProxyLoaderForm.AddProxys addProxys)
                : base(crawler, addProxys) { }
            public override void DoParse(CrawlUrl url)
            {
                this.Crawler.FetchContent(url);
                HtmlNode root = this.LoadHtml(url.Content);
                HtmlNode node = root.SelectSingleNode(@"//div[@class='cont_font']/p");
                if (node == null)
                {
                    Console.WriteLine(string.Format("node not found. url={0}", url.Url));
                    return;
                }
                List<WebProxy> proxys = new List<WebProxy>();
                foreach (string line in node.InnerHtml.Split(new string[] { @"<br>" }, StringSplitOptions.RemoveEmptyEntries))
                {
                    if (string.IsNullOrWhiteSpace(line))
                    { continue; }
                    Match match = HostPortRegex.Match(line);
                    if (match.Success)
                    {
                        string hostAndPort = match.Value;
                        string host = hostAndPort.Split(':')[0];
                        int port = Int32.Parse(hostAndPort.Split(':')[1]);
                        proxys.Add(new WebProxy(host, port));
                    }
                    else
                    {
                        Console.WriteLine("cannot parse line." + line);
                    }
                }
                AddProxys(proxys);
                HtmlNodeCollection listNodes = root.SelectNodes(@"//ul[@class='pagelist']/li/a[@href]");
                List<CrawlUrl> urls = new List<CrawlUrl>(); 
                foreach(HtmlNode listNode in listNodes)
                {
                    string a = listNode.Attributes["href"].Value;
                    urls.Add(new CrawlUrl("http://www.youdaili.net/Daili/guonei/"+listNode.Attributes["href"].Value));
                }
                Crawler.MyJobQueue.AddLasts(urls);
            }
        }
        #endregion

        #region 代理公用控件
        public abstract class ProxyLoaderHandlerBase : HtmlModuleHandlerBase
        {
            public string proxyFile;
            private event Proxys.ProxyLoaderForm.AddProxys addProxys;
           
            
            public ProxyLoaderHandlerBase(Crawler crawler, Proxys.ProxyLoaderForm.AddProxys addProxys)
                : base(crawler)
            {
                this.addProxys = addProxys;
                this.HostRegex = new Regex(@"[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+", RegexOptions.Compiled);
                this.HostPortRegex = new Regex(@"[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+", RegexOptions.Compiled);
            }

            public void AddProxys(List<WebProxy> proxys)
            {
                Console.WriteLine(string.Format("loaded. {0}:{1}", this.GetType().Name, proxys.Count));
                this.addProxys(proxys);
            }
            public Regex HostRegex { get; set; }
            public Regex HostPortRegex { get; set; }
        }
        #endregion

        #region proxy360
        public class Proxy360Module : HtmlModule
        {
            public override string ModuleName
            {
                get { return "proxy360"; }
            }

            public override bool Match(CrawlUrl url)
            {
                bool equals = url.Url.Equals(@"http://www.proxy360.cn/Region/China") || url.Url.Equals(@"http://www.proxy360.cn/Proxy") || url.Url.Equals(@"http://www.proxy360.cn/QQ-Proxy") || url.Url.Equals(@"http://www.proxy360.cn/MSN-Proxy");
                return equals;
            }
        }
        public class Proxy360Handler : ProxyLoaderHandlerBase
        {
            public Proxy360Handler(Crawler crawler, Proxys.ProxyLoaderForm.AddProxys addProxys)
                : base(crawler, addProxys)
            {
            }

            public override void DoParse(CrawlUrl url)
            {
                try
                {
                    this.Crawler.FetchContent(url);
                    HtmlNode root = this.LoadHtml(url.Content);
                    HtmlNodeCollection nodes = root.SelectNodes(@"//div[@name='list_proxy_ip']");
                    if (nodes == null)
                    { return; }
                    List<WebProxy> proxys = new List<WebProxy>();
                    foreach (HtmlNode node in nodes)
                    {
                        HtmlNodeCollection tdNodes = node.SelectNodes(@"div/span");
                        if (tdNodes != null && tdNodes.Count >= 8)
                        {
                            string host = tdNodes[0].InnerText.Trim();
                            int port = Int32.Parse(tdNodes[1].InnerText.Trim());
                            string area = tdNodes[3].InnerText.Trim();
                            if (!HostRegex.IsMatch(host) || !area.Contains("中国")) { continue; }
                            proxys.Add(new WebProxy(host, port));
                        }
                    }
                    this.AddProxys(proxys);
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.ToString());
                }
            }
        }
        #endregion
        
        #region proxy.com.ru
        public class ProxyRuModule : HtmlModule
        {
            public override string ModuleName
            {
                get { return "proxy.com.ru"; }
            }

            public override bool Match(CrawlUrl url)
            {
                return url.Url.ToLower().Contains(@"proxy.com.ru");
            }
        }
        public class ProxyRuHandler : ProxyLoaderHandlerBase
        {
            public ProxyRuHandler(Crawler crawler, Proxys.ProxyLoaderForm.AddProxys addProxys)
                : base(crawler, addProxys)
            {
            }
            public override void DoParse(CrawlUrl url)
            {
                try
                {
                    this.Crawler.FetchContent(url);
                    HtmlNode root = this.LoadHtml(url.Content);
                    HtmlNodeCollection nodes = root.SelectNodes(@"//body/center/font[1]//font[@size='2']/table/tr");
                    if (nodes == null)
                    { return; }
                    List<WebProxy> proxys = new List<WebProxy>();
                    foreach (HtmlNode node in nodes)
                    {
                        try
                        {
                            HtmlNodeCollection tdNodes = node.SelectNodes(@".//td");
                            if (tdNodes != null && tdNodes.Count >= 5)
                            {
                                string host = tdNodes[1].InnerText.Trim();
                                int port = Int32.Parse(tdNodes[2].InnerText.Trim());
                                string type = tdNodes[3].InnerText.Trim();
                                string area = tdNodes[4].InnerText.Trim();
                                if (!HostRegex.IsMatch(host)) { Console.WriteLine("host illegal." + host); continue; }
                                if (!AreaIllegal(area)) { Console.WriteLine("area illegal." + area); continue; }
                                //if (!type.Contains("匿名")) { Console.WriteLine("type illegal." + type); continue; }
                                proxys.Add(new WebProxy(host, port));
                            }
                        }
                        catch (Exception ex) { Console.WriteLine(ex.ToString()); }
                    }
                    this.AddProxys(proxys);
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.ToString());
                }
            }
            private bool AreaIllegal(string area)
            {
                foreach (string a in AREAS)
                { if (area.Contains(a)) { return true; } }
                return false;
            }
        }
        #endregion         

        #region 全国省市地址
        private static string[] AREAS = new string[]{"北京",
"浙江",
"天津",
"安徽",
"上海",
"福建",
"重庆",
"江西",
"香港",
"山东",
"澳门",
"河南",
"内蒙古",
"湖北",
"新疆",
"湖南",
"宁夏",
"广东",
"西藏",
"海南",
"广西",
"四川",
"河北",
"贵州",
"山西",
"云南",
"辽宁",
"陕西",
"吉林",
"甘肃",
"黑龙江",
"青海",
"江苏",
"台湾",
"东城",
"西城",
"崇文",
"宣武",
"朝阳",
"丰台",
"石景山",
"海淀",
"门头沟",
"房山",
"通州",
"顺义",
"昌平",
"大兴",
"怀柔",
"平谷",
"密云",
"延庆",
"黄浦",
"卢湾",
"徐汇",
"长宁",
"静安",
"普陀",
"闸北",
"虹口",
"杨浦",
"闵行",
"宝山",
"嘉定",
"浦东新",
"金山",
"松江",
"青浦",
"南汇",
"奉贤",
"崇明",
"和平",
"河东",
"河西",
"南开",
"红桥",
"塘沽",
"汉沽",
"大港",
"东丽",
"西青",
"津南",
"北辰",
"武清",
"宝坻",
"宁河",
"静海",
"蓟",
"万州",
"涪陵",
"渝中",
"大渡口",
"江北",
"沙坪坝",
"九龙坡",
"南岸",
"北碚",
"万盛",
"双桥",
"渝北",
"巴南",
"黔江",
"长寿",
"江津",
"合川",
"永川",
"南川",
"綦江",
"潼南",
"铜梁",
"大足",
"荣昌",
"璧山",
"梁平",
"城口",
"丰都",
"垫江",
"武隆",
"忠",
"开",
"云阳",
"奉节",
"巫山",
"巫溪",
"哈尔滨",
"齐齐哈尔",
"鸡西",
"鹤岗",
"双鸭山",
"大庆",
"伊春",
"佳木斯",
"七台河",
"牡丹江",
"黑河",
"绥化",
"大兴安岭地",
"长春",
"四平",
"辽源",
"通化",
"白山",
"松原",
"白城",
"延边朝鲜族自治州",
"沈阳",
"大连",
"鞍山",
"抚顺",
"本溪",
"丹东",
"锦州",
"营口",
"阜新",
"辽阳",
"盘锦",
"铁岭",
"葫芦岛",
"济南",
"青岛",
"淄博",
"枣庄",
"东营",
"烟台",
"潍坊",
"济宁",
"泰安",
"威海",
"日照",
"莱芜",
"临沂",
"德州",
"聊城",
"滨州",
"菏泽",
"太原",
"大同",
"阳泉",
"长治",
"晋城",
"朔州",
"晋中",
"运城",
"忻州",
"临汾",
"吕梁",
"西安",
"铜川",
"宝鸡",
"咸阳",
"渭南",
"延安",
"汉中",
"榆林",
"安康",
"商洛",
"石家庄",
"唐山",
"秦皇岛",
"邯郸",
"邢台",
"保定",
"张家口",
"承德",
"沧州",
"廊坊",
"衡水",
"郑州",
"开封",
"洛阳",
"平顶山",
"安阳",
"鹤壁",
"新乡",
"焦作",
"济源",
"濮阳",
"许昌",
"漯河",
"三门峡",
"南阳",
"商丘",
"信阳",
"周口",
"驻马店",
"武汉",
"黄石",
"十堰",
"宜昌",
"襄樊",
"鄂州",
"荆门",
"孝感",
"荆州",
"黄冈",
"咸宁",
"随州",
"仙桃",
"潜江",
"天门",
"神农架林",
"长沙",
"株洲",
"湘潭",
"衡阳",
"邵阳",
"岳阳",
"常德",
"张家界",
"益阳",
"郴州",
"永州",
"怀化",
"娄底",
"海口",
"三亚",
"五指山",
"琼海",
"儋州",
"文昌",
"万宁",
"东方",
"定安",
"屯昌",
"澄迈",
"临高",
"南京",
"无锡",
"徐州",
"常州",
"苏州",
"南通",
"连云港",
"淮安",
"盐城",
"扬州",
"镇江",
"泰州",
"宿迁",
"南昌",
"景德镇",
"萍乡",
"九江",
"新余",
"鹰潭",
"赣州",
"吉安",
"宜春",
"抚州",
"上饶",
"广州",
"韶关",
"深圳",
"珠海",
"汕头",
"佛山",
"江门",
"湛江",
"茂名",
"肇庆",
"惠州",
"梅州",
"汕尾",
"河源",
"阳江",
"清远",
"东莞",
"中山",
"潮州",
"揭阳",
"云浮",
"南宁",
"柳州",
"桂林",
"梧州",
"北海",
"防城港",
"钦州",
"贵港",
"玉林",
"百色",
"贺州",
"河池",
"来宾",
"崇左",
"昆明",
"曲靖",
"玉溪",
"保山",
"昭通",
"丽江",
"思茅",
"临沧",
"贵阳",
"六盘水",
"遵义",
"安顺",
"铜仁地",
"成都",
"自贡",
"攀枝花",
"泸州",
"德阳",
"绵阳",
"广元",
"遂宁",
"内江",
"乐山",
"南充",
"眉山",
"宜宾",
"广安",
"达州",
"雅安",
"巴中",
"资阳",
"呼和浩特",
"包头",
"乌海",
"赤峰",
"通辽",
"鄂尔多斯",
"呼伦贝尔",
"巴彦淖尔",
"乌兰察布",
"兴安盟",
"锡林郭勒盟",
"阿拉善盟",
"银川",
"石嘴山",
"吴忠",
"固原",
"中卫",
"兰州",
"嘉峪关",
"金昌",
"白银",
"天水",
"武威",
"张掖",
"平凉",
"酒泉",
"庆阳",
"定西",
"陇南",
"临夏回族自治州",
"甘南藏族自治州",
"西宁",
"海东地",
"海北藏族自治州",
"黄南藏族自治州",
"海南藏族自治州",
"果洛藏族自治州",
"玉树藏族自治州",
"海西蒙古族藏族自治州",
"拉萨",
"昌都地",
"山南地",
"日喀则地",
"那曲地",
"阿里地",
"林芝地",
"乌鲁木齐",
"合肥",
"芜湖",
"蚌埠",
"淮南",
"马鞍山",
"淮北",
"铜陵",
"安庆",
"黄山",
"滁州",
"阜阳",
"宿州",
"巢湖",
"六安",
"亳州",
"池州",
"宣城",
"杭州",
"宁波",
"温州",
"嘉兴",
"湖州",
"绍兴",
"金华",
"衢州",
"舟山",
"台州",
"丽水",
"福州",
"厦门",
"莆田",
"三明",
"泉州",
"漳州",
"南平",
"龙岩",
"宁德",
"台北",
"高雄",
"基隆",
"台中",
"台南",
"新竹",
"嘉义",
"中西",
"湾仔",
"东",
"南",
"油尖旺",
"深水埗",
"九龙城",
"黄大仙",
"观塘",
"荃湾",
"葵青",
"沙田",
"西贡",
"大埔",
"北",
"元朗",
"屯门",
"离岛",
};
        #endregion

        #region  http://cn-proxy.com/

        public class ProxyCnProxyModule : HtmlModule
        {
            public override string ModuleName
            {
                get { return "CnProxy"; }
            }

            public override bool Match(CrawlUrl url)
            {
                bool equals = url.Url.Equals(@"http://cn-proxy.com/");
                return equals;
            }
        }
        public class ProxyCnProxyHandler : ProxyLoaderHandlerBase
        {

            private Regex regexIpPort = new Regex(@"<td>(?<ip>\d{0,3}\.\d{0,3}\.\d{0,3}\.\d{0,3})</td>\n.*?<td>(?<port>\d+)</td>", RegexOptions.Multiline);
            public ProxyCnProxyHandler(Crawler crawler, Proxys.ProxyLoaderForm.AddProxys addProxys)
                : base(crawler, addProxys)
            {
            }

            public override void DoParse(CrawlUrl url)
            {
                try
                {
                    this.Crawler.FetchContent(url);
                
                    List<WebProxy> proxys = new List<WebProxy>();
                    
                    MatchCollection mc =  regexIpPort.Matches(url.Content);
                    if (mc.Count > 0) 
                    {
                        foreach (Match m in mc)
                        {
                            string host = m.Groups["ip"].Value;;
                            int port = Int32.Parse(m.Groups["port"].Value);
                            proxys.Add(new WebProxy(host, port));
                        }

                    }

                    this.AddProxys(proxys);
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.ToString());
                }
            }


        }
#endregion

    }
    #endregion
}
